Mercurial > fix-ceb-nulls / changeset

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Dockerfile	Mon Aug 01 11:29:27 2022 +0100
@@ -0,0 +1,18 @@
+FROM alpine AS build
+
+RUN apk add openssl3-dev zlib-dev lua5.3-dev musl-dev luarocks gcc git \
+ && luarocks-5.3 install luaossl \
+ && luarocks-5.3 install lua-zlib
+
+FROM alpine
+
+RUN apk add libssl3 libcrypto3 zlib lua5.3 luarocks
+
+COPY --from=build /usr/local/ /usr/local/
+
+ADD fix-ceb-nulls.lua /usr/local/bin/fix-ceb-nulls
+RUN chmod 750 /usr/local/bin/fix-ceb-nulls
+
+ENTRYPOINT ["/usr/local/bin/fix-ceb-nulls"]
+
+WORKDIR /data

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Makefile	Mon Aug 01 11:29:27 2022 +0100
@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t mwild1/fix-ceb-nulls .

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/README.md	Mon Aug 01 11:29:27 2022 +0100
@@ -0,0 +1,40 @@
+This utility strips NUL bytes from Conversations backup (.ceb) files. For
+unknown reasons, backups from certain Android versions/devices seem to end up
+with these erroneous bytes, and then fail to import in the app.
+
+## Usage
+
+### Run via docker
+
+This option works on any system where docker is installed, and does not require
+anything else to be installed on the host system.
+
+```
+docker run --rm -v "$PWD:/data" mwild1/fix-ceb-nulls BACKUP_FILE_NAME BACKUP_FILE_PASSWORD
+```
+
+The *BACKUP_FILE_NAME* should be in the current directory where you run the
+command. The result will also be written to the current directory, with the
+suffix `-fixed.ceb`.
+
+### Run manually
+
+The script has a few dependencies, but they shouldn't be too hard to install
+on most systems.
+
+Dependencies:
+
+- Lua 5.3
+- lua-zlib (luarocks: lua-zlib, Debian/Ubuntu: lua-zlib)
+- luaossl 2022 (luarocks: luaossl, Debian/Ubuntu: lua-luaossl (⚠️ at the time of
+    writing this, Debian/Ubuntu do not have the latest version available,
+    install via luarocks instead))
+
+Run:
+
+```
+lua5.3 fix-ceb-nulls.lua BACKUP_FILE_NAME BACKUP_FILE_PASSWORD
+```
+
+The result will be written to a new file in the same location as the original
+file, but with a `-fixed.ceb` suffix.

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/fix-ceb-nulls.lua	Mon Aug 01 11:29:27 2022 +0100
@@ -0,0 +1,165 @@
+#!/usr/bin/env lua5.3
+
+if not _VERSION:match("^Lua 5%.[34]") then
+	print("This utility requires Lua 5.3/5.4");
+	return 1;
+end
+
+local have_ciphers, ciphers = pcall(require, "openssl.cipher");
+if not have_ciphers then
+	print("openssl.ciphers module not found.");
+	print("On Debian, install lua-luaossl, or install luarocks and run");
+	print("  luarocks install luaossl");
+	print("");
+end
+
+local have_kdf, kdf = pcall(require, "openssl.kdf");
+if not have_kdf then
+	print("openssl.kdf module not found.");
+	print("On Debian, install lua-luaossl, or install luarocks and run");
+	print("  luarocks install luaossl");
+	print("");
+end
+
+local have_zlib, zlib = pcall(require, "zlib");
+if not have_zlib then
+	print("zlib module not found.");
+	print("On Debian, install lua-zlib, or install luarocks and run");
+	print("  luarocks install lua-zlib");
+	print("");
+end
+
+if #arg == 0 then
+	print("Fix Conversations backup files that contain NUL bytes");
+	print("");
+	print("Usage:");
+	print("");
+	print("", arg[0].." INPUT_FILE PASSWORD");
+	print("");
+	print("On success, a new backup file will be created with the suffix '-fixed.ceb'.");
+	print("");
+end
+
+if #arg == 0 or not have_zlib or not have_kdf or not have_ciphers then
+	return 1;
+end
+
+local input_filename = assert(arg[1], "no ceb file specified");
+
+local file_password = assert(arg[2], "no password specified");
+
+local function read_header(f)
+	local function read_int()
+		return (">i4"):unpack(f:read(4));
+	end
+	local function read_short()
+		return (">i2"):unpack(f:read(2));
+	end
+	local function read_long()
+		return (">i8"):unpack(f:read(8));
+	end
+	local function read_string()
+		local n = read_short();
+		return f:read(n);
+	end
+
+	return {
+		version = read_int();
+		app_id = read_string();
+		jid = read_string();
+		timestamp = math.floor(read_long()/1000);
+		iv = f:read(12);
+		salt = f:read(16);
+	};
+end
+
+local function write_header(f, header)
+	local function write_int(f, n)
+		f:write((">i4"):pack(n));
+	end
+	local function write_short(f, n)
+		f:write((">i2"):pack(n));
+	end
+	local function write_long(f, n)
+		f:write((">i8"):pack(n));
+	end
+	local function write_string(f, s)
+		write_short(f, #s);
+		f:write(s);
+	end
+
+	write_int(f, header.version);
+	write_string(f, header.app_id);
+	write_string(f, header.jid);
+	write_long(f, header.timestamp*1000);
+	assert(#header.iv == 12);
+	f:write(header.iv);
+	assert(#header.salt == 16);
+	f:write(header.salt);
+end
+
+local f = io.open(input_filename);
+
+local header = read_header(f);
+
+print("version", header.version);
+print("app", header.app_id);
+print("jid", header.jid);
+print("timestamp", os.date("%c", header.timestamp));
+
+
+local function generate_key(password, salt)
+	return kdf.derive({
+		type = "PBKDF2";
+		md = "sha1";
+		pass = password;
+		salt = salt;
+		iter = 1024;
+		outlen = 128/8;
+	});
+end
+
+print("k", #(generate_key(file_password, header.salt)));
+
+local decryption_key = generate_key(file_password, header.salt);
+
+local cipher = ciphers.new("AES-128-GCM"):decrypt(decryption_key, header.iv);
+
+local decompress = zlib.inflate();
+
+local compress = zlib.deflate();
+local cipher_out = ciphers.new("AES-128-GCM"):encrypt(decryption_key, header.iv);
+
+local null_replacement = string.char(0xE2, 0x90, 0x80); -- U+2400 SYMBOL FOR NULL
+
+local outfile = assert(io.open(input_filename:gsub("%.ceb$", "-fixed.ceb"), "w+"));
+write_header(outfile, header);
+
+repeat
+	local enc_data = f:read(4096);
+	if not enc_data then break; end
+
+	local gz_data = assert(cipher:update(enc_data));
+
+	local status, data, eof = pcall(decompress, gz_data);
+	if not status then
+		print("EE: Failed to decompress: "..tostring(data));
+		return 1;
+	end
+
+	local status, recompressed = pcall(compress, ((data:gsub("\000", null_replacement))));
+	if not status then
+		print("EE: Failed to compress: "..tostring(data));
+		return 1;
+	end
+	local reencrypted = cipher_out:update(recompressed);
+	outfile:write(reencrypted);
+until eof
+
+local final_data = cipher_out:final();
+if final_data and final_data ~= "" then
+	outfile:write(final_data);
+end
+outfile:close();
+
+print("Done");