Merge pull request #38308 from bruvzg/sad_security_circus

Adds PCK encryption support (using script encryption key for export).
This commit is contained in:
Rémi Verschelde 2020-09-07 17:12:01 +02:00 committed by GitHub
commit 3c42d5789f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 692 additions and 151 deletions

View file

@ -37,52 +37,54 @@
#include <stdio.h>
#define COMP_MAGIC 0x43454447
Error FileAccessEncrypted::open_and_parse(FileAccess *p_base, const Vector<uint8_t> &p_key, Mode p_mode) {
Error FileAccessEncrypted::open_and_parse(FileAccess *p_base, const Vector<uint8_t> &p_key, Mode p_mode, bool p_with_magic) {
ERR_FAIL_COND_V_MSG(file != nullptr, ERR_ALREADY_IN_USE, "Can't open file while another file from path '" + file->get_path_absolute() + "' is open.");
ERR_FAIL_COND_V(p_key.size() != 32, ERR_INVALID_PARAMETER);
pos = 0;
eofed = false;
use_magic = p_with_magic;
if (p_mode == MODE_WRITE_AES256) {
data.clear();
writing = true;
file = p_base;
mode = p_mode;
key = p_key;
} else if (p_mode == MODE_READ) {
writing = false;
key = p_key;
uint32_t magic = p_base->get_32();
ERR_FAIL_COND_V(magic != COMP_MAGIC, ERR_FILE_UNRECOGNIZED);
mode = Mode(p_base->get_32());
ERR_FAIL_INDEX_V(mode, MODE_MAX, ERR_FILE_CORRUPT);
ERR_FAIL_COND_V(mode == 0, ERR_FILE_CORRUPT);
if (use_magic) {
uint32_t magic = p_base->get_32();
ERR_FAIL_COND_V(magic != ENCRYPTED_HEADER_MAGIC, ERR_FILE_UNRECOGNIZED);
}
unsigned char md5d[16];
p_base->get_buffer(md5d, 16);
length = p_base->get_64();
unsigned char iv[16];
for (int i = 0; i < 16; i++) {
iv[i] = p_base->get_8();
}
base = p_base->get_position();
ERR_FAIL_COND_V(p_base->get_len() < base + length, ERR_FILE_CORRUPT);
uint32_t ds = length;
if (ds % 16) {
ds += 16 - (ds % 16);
}
data.resize(ds);
uint32_t blen = p_base->get_buffer(data.ptrw(), ds);
ERR_FAIL_COND_V(blen != ds, ERR_FILE_CORRUPT);
CryptoCore::AESContext ctx;
ctx.set_decode_key(key.ptrw(), 256);
{
CryptoCore::AESContext ctx;
for (size_t i = 0; i < ds; i += 16) {
ctx.decrypt_ecb(&data.write[i], &data.write[i]);
ctx.set_encode_key(key.ptrw(), 256); // Due to the nature of CFB, same key schedule is used for both encryption and decryption!
ctx.decrypt_cfb(ds, iv, data.ptrw(), data.ptrw());
}
data.resize(length);
@ -119,6 +121,25 @@ void FileAccessEncrypted::close() {
return;
}
_release();
file->close();
memdelete(file);
file = nullptr;
}
void FileAccessEncrypted::release() {
if (!file) {
return;
}
_release();
file = nullptr;
}
void FileAccessEncrypted::_release() {
if (writing) {
Vector<uint8_t> compressed;
size_t len = data.size();
@ -138,27 +159,23 @@ void FileAccessEncrypted::close() {
CryptoCore::AESContext ctx;
ctx.set_encode_key(key.ptrw(), 256);
for (size_t i = 0; i < len; i += 16) {
ctx.encrypt_ecb(&compressed.write[i], &compressed.write[i]);
if (use_magic) {
file->store_32(ENCRYPTED_HEADER_MAGIC);
}
file->store_32(COMP_MAGIC);
file->store_32(mode);
file->store_buffer(hash, 16);
file->store_64(data.size());
file->store_buffer(compressed.ptr(), compressed.size());
file->close();
memdelete(file);
file = nullptr;
data.clear();
unsigned char iv[16];
for (int i = 0; i < 16; i++) {
iv[i] = Math::rand() % 256;
file->store_8(iv[i]);
}
} else {
file->close();
memdelete(file);
ctx.encrypt_cfb(len, iv, compressed.ptrw(), compressed.ptrw());
file->store_buffer(compressed.ptr(), compressed.size());
data.clear();
file = nullptr;
}
}

View file

@ -33,6 +33,8 @@
#include "core/os/file_access.h"
#define ENCRYPTED_HEADER_MAGIC 0x43454447
class FileAccessEncrypted : public FileAccess {
public:
enum Mode {
@ -42,7 +44,6 @@ public:
};
private:
Mode mode = MODE_MAX;
Vector<uint8_t> key;
bool writing = false;
FileAccess *file = nullptr;
@ -51,13 +52,17 @@ private:
Vector<uint8_t> data;
mutable int pos = 0;
mutable bool eofed = false;
bool use_magic = true;
void _release();
public:
Error open_and_parse(FileAccess *p_base, const Vector<uint8_t> &p_key, Mode p_mode);
Error open_and_parse(FileAccess *p_base, const Vector<uint8_t> &p_key, Mode p_mode, bool p_with_magic = true);
Error open_and_parse_password(FileAccess *p_base, const String &p_key, Mode p_mode);
virtual Error _open(const String &p_path, int p_mode_flags); ///< open a file
virtual void close(); ///< close a file
virtual void release(); ///< finish and keep base file open
virtual bool is_open() const; ///< true when file is open
virtual String get_path() const; /// returns the path for the current open file

View file

@ -30,6 +30,8 @@
#include "file_access_pack.h"
#include "core/io/file_access_encrypted.h"
#include "core/script_language.h"
#include "core/version.h"
#include <stdio.h>
@ -44,13 +46,14 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, size_t p_
return ERR_FILE_UNRECOGNIZED;
}
void PackedData::add_path(const String &pkg_path, const String &path, uint64_t ofs, uint64_t size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files) {
void PackedData::add_path(const String &pkg_path, const String &path, uint64_t ofs, uint64_t size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted) {
PathMD5 pmd5(path.md5_buffer());
//printf("adding path %s, %lli, %lli\n", path.utf8().get_data(), pmd5.a, pmd5.b);
bool exists = files.has(pmd5);
PackedFile pf;
pf.encrypted = p_encrypted;
pf.pack = pkg_path;
pf.offset = ofs;
pf.size = size;
@ -179,6 +182,11 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
ERR_FAIL_V_MSG(false, "Pack created with a newer version of the engine: " + itos(ver_major) + "." + itos(ver_minor) + ".");
}
uint32_t pack_flags = f->get_32();
uint64_t file_base = f->get_64();
bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED);
for (int i = 0; i < 16; i++) {
//reserved
f->get_32();
@ -186,6 +194,30 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
int file_count = f->get_32();
if (enc_directory) {
FileAccessEncrypted *fae = memnew(FileAccessEncrypted);
if (!fae) {
f->close();
memdelete(f);
ERR_FAIL_V_MSG(false, "Can't open encrypted pack directory.");
}
Vector<uint8_t> key;
key.resize(32);
for (int i = 0; i < key.size(); i++) {
key.write[i] = script_encryption_key[i];
}
Error err = fae->open_and_parse(f, key, FileAccessEncrypted::MODE_READ, false);
if (err) {
f->close();
memdelete(f);
memdelete(fae);
ERR_FAIL_V_MSG(false, "Can't open encrypted pack directory.");
}
f = fae;
}
for (int i = 0; i < file_count; i++) {
uint32_t sl = f->get_32();
CharString cs;
@ -196,11 +228,13 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
String path;
path.parse_utf8(cs.ptr());
uint64_t ofs = f->get_64();
uint64_t ofs = file_base + f->get_64();
uint64_t size = f->get_64();
uint8_t md5[16];
f->get_buffer(md5, 16);
PackedData::get_singleton()->add_path(p_path, path, ofs + p_offset, size, md5, this, p_replace_files);
uint32_t flags = f->get_32();
PackedData::get_singleton()->add_path(p_path, path, ofs + p_offset, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED));
}
f->close();
@ -234,7 +268,7 @@ void FileAccessPack::seek(size_t p_position) {
eof = false;
}
f->seek(pf.offset + p_position);
f->seek(off + p_position);
pos = p_position;
}
@ -319,12 +353,35 @@ FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFil
ERR_FAIL_COND_MSG(!f, "Can't open pack-referenced file '" + String(pf.pack) + "'.");
f->seek(pf.offset);
off = pf.offset;
if (pf.encrypted) {
FileAccessEncrypted *fae = memnew(FileAccessEncrypted);
if (!fae) {
ERR_FAIL_MSG("Can't open encrypted pack-referenced file '" + String(pf.pack) + "'.");
}
Vector<uint8_t> key;
key.resize(32);
for (int i = 0; i < key.size(); i++) {
key.write[i] = script_encryption_key[i];
}
Error err = fae->open_and_parse(f, key, FileAccessEncrypted::MODE_READ, false);
if (err) {
memdelete(fae);
ERR_FAIL_MSG("Can't open encrypted pack-referenced file '" + String(pf.pack) + "'.");
}
f = fae;
off = 0;
}
pos = 0;
eof = false;
}
FileAccessPack::~FileAccessPack() {
if (f) {
f->close();
memdelete(f);
}
}

View file

@ -40,7 +40,15 @@
// Godot's packed file magic header ("GDPC" in ASCII).
#define PACK_HEADER_MAGIC 0x43504447
// The current packed file format version number.
#define PACK_FORMAT_VERSION 1
#define PACK_FORMAT_VERSION 2
enum PackFlags {
PACK_DIR_ENCRYPTED = 1 << 0
};
enum PackFileFlags {
PACK_FILE_ENCRYPTED = 1 << 0
};
class PackSource;
@ -56,6 +64,7 @@ public:
uint64_t size;
uint8_t md5[16];
PackSource *src;
bool encrypted;
};
private:
@ -102,7 +111,7 @@ private:
public:
void add_pack_source(PackSource *p_source);
void add_path(const String &pkg_path, const String &path, uint64_t ofs, uint64_t size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files); // for PackSource
void add_path(const String &pkg_path, const String &path, uint64_t ofs, uint64_t size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false); // for PackSource
void set_disabled(bool p_disabled) { disabled = p_disabled; }
_FORCE_INLINE_ bool is_disabled() const { return disabled; }
@ -135,6 +144,7 @@ class FileAccessPack : public FileAccess {
mutable size_t pos;
mutable bool eof;
uint64_t off;
FileAccess *f;
virtual Error _open(const String &p_path, int p_mode_flags);

View file

@ -200,7 +200,7 @@ bool ZipArchive::try_open_pack(const String &p_path, bool p_replace_files, size_
files[fname] = f;
uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
PackedData::get_singleton()->add_path(p_path, fname, 1, 0, md5, this, p_replace_files);
PackedData::get_singleton()->add_path(p_path, fname, 1, 0, md5, this, p_replace_files, false);
//printf("packed data add path %s, %s\n", p_name.utf8().get_data(), fname.utf8().get_data());
if ((i + 1) < gi.number_entry) {

View file

@ -30,36 +30,58 @@
#include "pck_packer.h"
#include "core/crypto/crypto_core.h"
#include "core/io/file_access_encrypted.h"
#include "core/io/file_access_pack.h" // PACK_HEADER_MAGIC, PACK_FORMAT_VERSION
#include "core/os/file_access.h"
#include "core/version.h"
static uint64_t _align(uint64_t p_n, int p_alignment) {
if (p_alignment == 0) {
return p_n;
static int _get_pad(int p_alignment, int p_n) {
int rest = p_n % p_alignment;
int pad = 0;
if (rest > 0) {
pad = p_alignment - rest;
}
uint64_t rest = p_n % p_alignment;
if (rest == 0) {
return p_n;
} else {
return p_n + (p_alignment - rest);
}
}
static void _pad(FileAccess *p_file, int p_bytes) {
for (int i = 0; i < p_bytes; i++) {
p_file->store_8(0);
}
return pad;
}
void PCKPacker::_bind_methods() {
ClassDB::bind_method(D_METHOD("pck_start", "pck_name", "alignment"), &PCKPacker::pck_start, DEFVAL(0));
ClassDB::bind_method(D_METHOD("add_file", "pck_path", "source_path"), &PCKPacker::add_file);
ClassDB::bind_method(D_METHOD("pck_start", "pck_name", "alignment", "key", "encrypt_directory"), &PCKPacker::pck_start, DEFVAL(0), DEFVAL(String()), DEFVAL(false));
ClassDB::bind_method(D_METHOD("add_file", "pck_path", "source_path", "encrypt"), &PCKPacker::add_file, DEFVAL(false));
ClassDB::bind_method(D_METHOD("flush", "verbose"), &PCKPacker::flush, DEFVAL(false));
}
Error PCKPacker::pck_start(const String &p_file, int p_alignment) {
Error PCKPacker::pck_start(const String &p_file, int p_alignment, const String &p_key, bool p_encrypt_directory) {
ERR_FAIL_COND_V_MSG((p_key.empty() || !p_key.is_valid_hex_number(false) || p_key.length() != 64), ERR_CANT_CREATE, "Invalid Encryption Key (must be 64 characters long).");
String _key = p_key.to_lower();
key.resize(32);
for (int i = 0; i < 32; i++) {
int v = 0;
if (i * 2 < _key.length()) {
char32_t ct = _key[i * 2];
if (ct >= '0' && ct <= '9') {
ct = ct - '0';
} else if (ct >= 'a' && ct <= 'f') {
ct = 10 + ct - 'a';
}
v |= ct << 4;
}
if (i * 2 + 1 < _key.length()) {
char32_t ct = _key[i * 2 + 1];
if (ct >= '0' && ct <= '9') {
ct = ct - '0';
} else if (ct >= 'a' && ct <= 'f') {
ct = 10 + ct - 'a';
}
v |= ct;
}
key.write[i] = v;
}
enc_dir = p_encrypt_directory;
if (file != nullptr) {
memdelete(file);
}
@ -76,16 +98,19 @@ Error PCKPacker::pck_start(const String &p_file, int p_alignment) {
file->store_32(VERSION_MINOR);
file->store_32(VERSION_PATCH);
for (int i = 0; i < 16; i++) {
file->store_32(0); // reserved
uint32_t pack_flags = 0;
if (enc_dir) {
pack_flags |= PACK_DIR_ENCRYPTED;
}
file->store_32(pack_flags); // flags
files.clear();
ofs = 0;
return OK;
}
Error PCKPacker::add_file(const String &p_file, const String &p_src) {
Error PCKPacker::add_file(const String &p_file, const String &p_src, bool p_encrypt) {
FileAccess *f = FileAccess::open(p_src, FileAccess::READ);
if (!f) {
return ERR_FILE_CANT_OPEN;
@ -94,8 +119,32 @@ Error PCKPacker::add_file(const String &p_file, const String &p_src) {
File pf;
pf.path = p_file;
pf.src_path = p_src;
pf.ofs = ofs;
pf.size = f->get_len();
pf.offset_offset = 0;
Vector<uint8_t> data = FileAccess::get_file_as_array(p_src);
{
unsigned char hash[16];
CryptoCore::md5(data.ptr(), data.size(), hash);
pf.md5.resize(16);
for (int i = 0; i < 16; i++) {
pf.md5.write[i] = hash[i];
}
}
pf.encrypted = p_encrypt;
uint64_t _size = pf.size;
if (p_encrypt) { // Add encryption overhead.
if (_size % 16) { // Pad to encryption block size.
_size += 16 - (_size % 16);
}
_size += 16; // hash
_size += 8; // data size
_size += 16; // iv
}
int pad = _get_pad(alignment, ofs + _size);
ofs = ofs + _size + pad;
files.push_back(pf);
@ -108,27 +157,64 @@ Error PCKPacker::add_file(const String &p_file, const String &p_src) {
Error PCKPacker::flush(bool p_verbose) {
ERR_FAIL_COND_V_MSG(!file, ERR_INVALID_PARAMETER, "File must be opened before use.");
// write the index
int64_t file_base_ofs = file->get_position();
file->store_64(0); // files base
file->store_32(files.size());
for (int i = 0; i < files.size(); i++) {
file->store_pascal_string(files[i].path);
files.write[i].offset_offset = file->get_position();
file->store_64(0); // offset
file->store_64(files[i].size); // size
// # empty md5
file->store_32(0);
file->store_32(0);
file->store_32(0);
file->store_32(0);
for (int i = 0; i < 16; i++) {
file->store_32(0); // reserved
}
uint64_t ofs = file->get_position();
ofs = _align(ofs, alignment);
// write the index
file->store_32(files.size());
_pad(file, ofs - file->get_position());
FileAccessEncrypted *fae = nullptr;
FileAccess *fhead = file;
if (enc_dir) {
fae = memnew(FileAccessEncrypted);
ERR_FAIL_COND_V(!fae, ERR_CANT_CREATE);
Error err = fae->open_and_parse(file, key, FileAccessEncrypted::MODE_WRITE_AES256, false);
ERR_FAIL_COND_V(err != OK, ERR_CANT_CREATE);
fhead = fae;
}
for (int i = 0; i < files.size(); i++) {
int string_len = files[i].path.utf8().length();
int pad = _get_pad(4, string_len);
fhead->store_32(string_len + pad);
fhead->store_buffer((const uint8_t *)files[i].path.utf8().get_data(), string_len);
for (int j = 0; j < pad; j++) {
fhead->store_8(0);
}
fhead->store_64(files[i].ofs);
fhead->store_64(files[i].size); // pay attention here, this is where file is
fhead->store_buffer(files[i].md5.ptr(), 16); //also save md5 for file
uint32_t flags = 0;
if (files[i].encrypted) {
flags |= PACK_FILE_ENCRYPTED;
}
fhead->store_32(flags);
}
if (fae) {
fae->release();
memdelete(fae);
}
int header_padding = _get_pad(alignment, file->get_position());
for (int i = 0; i < header_padding; i++) {
file->store_8(Math::rand() % 256);
}
int64_t file_base = file->get_position();
file->seek(file_base_ofs);
file->store_64(file_base); // update files base
file->seek(file_base);
const uint32_t buf_max = 65536;
uint8_t *buf = memnew_arr(uint8_t, buf_max);
@ -137,26 +223,41 @@ Error PCKPacker::flush(bool p_verbose) {
for (int i = 0; i < files.size(); i++) {
FileAccess *src = FileAccess::open(files[i].src_path, FileAccess::READ);
uint64_t to_write = files[i].size;
fae = nullptr;
FileAccess *ftmp = file;
if (files[i].encrypted) {
fae = memnew(FileAccessEncrypted);
ERR_FAIL_COND_V(!fae, ERR_CANT_CREATE);
Error err = fae->open_and_parse(file, key, FileAccessEncrypted::MODE_WRITE_AES256, false);
ERR_FAIL_COND_V(err != OK, ERR_CANT_CREATE);
ftmp = fae;
}
while (to_write > 0) {
int read = src->get_buffer(buf, MIN(to_write, buf_max));
file->store_buffer(buf, read);
ftmp->store_buffer(buf, read);
to_write -= read;
}
uint64_t pos = file->get_position();
file->seek(files[i].offset_offset); // go back to store the file's offset
file->store_64(ofs);
file->seek(pos);
if (fae) {
fae->release();
memdelete(fae);
}
ofs = _align(ofs + files[i].size, alignment);
_pad(file, ofs - pos);
int pad = _get_pad(alignment, file->get_position());
for (int j = 0; j < pad; j++) {
file->store_8(Math::rand() % 256);
}
src->close();
memdelete(src);
count += 1;
if (p_verbose && files.size() > 0) {
const int file_num = files.size();
if (p_verbose && (file_num > 0)) {
if (count % 100 == 0) {
printf("%i/%i (%.2f)\r", count, files.size(), float(count) / files.size() * 100);
printf("%i/%i (%.2f)\r", count, file_num, float(count) / file_num * 100);
fflush(stdout);
}
}

View file

@ -40,20 +40,26 @@ class PCKPacker : public Reference {
FileAccess *file = nullptr;
int alignment;
uint64_t ofs = 0;
Vector<uint8_t> key;
bool enc_dir = false;
static void _bind_methods();
struct File {
String path;
String src_path;
int size;
uint64_t offset_offset;
uint64_t ofs;
uint64_t size;
bool encrypted;
Vector<uint8_t> md5;
};
Vector<File> files;
public:
Error pck_start(const String &p_file, int p_alignment = 0);
Error add_file(const String &p_file, const String &p_src);
Error pck_start(const String &p_file, int p_alignment = 0, const String &p_key = String(), bool p_encrypt_directory = false);
Error add_file(const String &p_file, const String &p_src, bool p_encrypt = false);
Error flush(bool p_verbose = false);
PCKPacker() {}